/*
 * REFXMLDtdParser.cpp
 *
 *  Created on: Apr 1, 2013
 *      Author: Lucifer
 */

#include "../../pch.h"
#include "../core/XMLCharType.h"
#include "REFXMLDtdParser.h"
#include "../../unicode/CharUnicodeInfo.h"

namespace BFX
{

REFXMLDtdParser::REFXMLDtdParser(REFXMLDtdParserAdapter* pReaderAdapter)
{
	BFX_ASSERT(pReaderAdapter != NULL);
	m_pReaderAdapter = pReaderAdapter;
	m_pNameTable = pReaderAdapter->GetNameTable();

	m_nExternalEntitiesDepth	= 0;
	m_nCurEntityId	= 0;
	m_nNextEntityId	= 1;

	m_bSupportNamespaces	= false;
	m_bNormalize	= false;
	m_bV1Compat		= false;
	m_bValidate		= false;

	m_nCondSectionDepth = 0;

	m_nTokenStartPos = 0;
	m_nColonPos = 0;
	m_pChars = NULL;
	m_nCharsUsed = 0;
	m_nCurPos = 0;
	m_scanningFunction = ScanningFunctionSubsetContent;
	m_nextScaningFunction = ScanningFunctionSubsetContent;
	m_savedScanningFunction = ScanningFunctionSubsetContent;
	m_bWhitespaceBeforeToken = false;

	m_pSchemaInfo = new REFXMLSchemaInfo();
	m_pSchemaInfo->SetSchemaType(REFXMLSchemaDecl::SchemaTypeDTD);

	m_pStringBuilder = new REFUniStringBuilder();
	m_pInternalSubsetValueSb = new REFUniStringBuilder();

	m_bFreeFloatingDtd = false;
	m_bHasFreeFloatingInternalSubset = false;
	m_literalQuoteChar = '"';
}

REFXMLDtdParser::~REFXMLDtdParser()
{
}

//
// Scanned data getters
//
REFXMLQualifiedName* REFXMLDtdParser::GetNameQualified(bool bCanHavePrefix)
{
	BFX_ASSERT(m_nCurPos - m_nTokenStartPos > 0);
	REF < REFXMLQualifiedName > pName;
	if (m_nColonPos == -1)
	{
		pName = new REFXMLQualifiedName(
				m_pNameTable->Add(UniString(m_pChars + m_nTokenStartPos, m_nCurPos - m_nTokenStartPos)));
	}
	else
	{
		if (bCanHavePrefix)
		{
			pName = new REFXMLQualifiedName(
					m_pNameTable->Add(UniString(m_pChars + m_nColonPos + 1, m_nCurPos - m_nColonPos - 1)),
					m_pNameTable->Add(UniString(m_pChars + m_nTokenStartPos, m_nCurPos - m_nTokenStartPos)));
		}
		else
		{
			SetLastError(XML_ColonInLocalName, CSTR(GetNameString()), m_nTokenStartPos);
			pName = NULL;
		}
	}

	if (pName != NULL)
		pName->Autorelease(); // NOTE: we have to put it into autorelease pool.
	return pName;
}
bool REFXMLDtdParser::Parse(bool bSaveSubset)
{
	bool bSuccess;
	if (m_bFreeFloatingDtd)
	{
		bSuccess = ParseFreeFloatingDtd();
	}
	else
	{
		bSuccess = ParseInDocumentDtd(bSaveSubset);
	}

	m_pSchemaInfo->Finish();

	// check undeclared forward references
	if (bSuccess && m_bValidate && m_pUndeclaredNotations != NULL)
	{
		for (UndeclaredNotationMap::POSITION entry = m_pUndeclaredNotations->GetFirstEntry();
			entry != NULL; entry = m_pUndeclaredNotations->GetNextEntry(entry))
		{
			UndeclaredNotation* un = entry->value;
			while (un != NULL)
			{
				// TODO: validate
				BFX_ASSERT(false);
				un = un->m_pNext;
			}
		}
		BFX_ASSERT(false);
	}
	return bSuccess;
}

bool REFXMLDtdParser::ParseFreeFloatingDtd()
{
	if (m_bHasFreeFloatingInternalSubset)
	{
		LoadParsingBuffer();
		if (!ParseInternalSubset())
			return false;
		SaveParsingBuffer();
	}
	if (!m_strSystemId.IsEmpty())
	{
		if (!ParseExternalSubset())
			return false;
	}
	return true;
}

bool REFXMLDtdParser::ParseInDocumentDtd(bool bSaveInternalSubset)
{
	LoadParsingBuffer();

	m_scanningFunction = ScanningFunctionQName;
	m_nextScaningFunction = ScanningFunctionDoctype1;

	// doctype name
	Token token = GetToken(false);
	if (token != TokenQName)
	{
		if (token != TokenError)
		{
			BFX_ENSURE((token == TokenQName)
					&& "This is an unexpected error that should have been handled in the ScanXXX methods.");
			SetLastError (XML_InternalError);
		}
		return false;
	}

	REFXMLQualifiedName* pDocTypeName = GetNameQualified(true);
	if (pDocTypeName == NULL)
		return false;
	m_pSchemaInfo->SetDocTypeName(pDocTypeName);

	// SYSTEM or PUBLIC id
	token = GetToken(false);
	if (token == TokenSYSTEM || token == TokenPUBLIC)
	{
		if (!ParseExternalId(token, TokenDOCTYPE, OUT m_strPublicId, OUT m_strSystemId))
			return false;
		token = GetToken(false);
	}

	switch (token)
	{
	case TokenLeftBracket:
		if (bSaveInternalSubset)
		{
			SaveParsingBuffer(); // this will cause saving the internal subset right from the point after '['
			m_pInternalSubsetValueSb->SetLength(0);
		}
		if (!ParseInternalSubset())
		{
			return false;
		}
		break;
	case TokenGreaterThan:
		break;
	case TokenError:
		return false;
	default:
		BFX_ASSERT(false
				&& "This is an unexpected error that should have been handled in the ScanXXX methods.");
		SetLastError (XML_InternalError);
		return false;
	}
	SaveParsingBuffer();

	if (!m_strSystemId.IsEmpty())
	{
		if (!ParseExternalSubset())
			return false;
	}

	return true;
}

// NOTE: not completely implement.
bool REFXMLDtdParser::ParseSubset()
{
	int nStartTagEntityId;
	for (;;)
	{
		Token token = GetToken(false);
		nStartTagEntityId = m_nCurEntityId;
		switch (token)
		{
		case TokenAttlistDecl:
			ParseAttlistDecl();
			break;

		case TokenElementDecl:
			ParseElementDecl();
			break;

		case TokenEntityDecl:
			ParseEntityDecl();
			break;

		case TokenNotationDecl:
			BFX_ASSERT(false);
			ParseNotationDecl();
			break;

		case TokenComment:
			ParseComment();
			break;

		case TokenPI:
			ParsePI();
			break;

		case TokenCondSectionStart:
			if (IsParsingInternalSubset())
			{
				// 3==strlen("<![")
				SetLastError(XML_InvalidConditionalSection, m_nCurPos - 3);
				return false;
			}
			ParseCondSection();
			nStartTagEntityId = m_nCurEntityId;
			break;
		case TokenCondSectionEnd:
			if (m_nCondSectionDepth > 0)
			{
				m_nCondSectionDepth--;
				if (m_bValidate /*????*/)
				{
					// TODO: validate.
					BFX_ASSERT(false);
				}
			}
			else
			{
				SetLastError(XML_UnexpectedCDataEnd, m_nCurPos - 3);
				return false;
			}
			break;
		case TokenRightBracket:
			if (IsParsingInternalSubset())
			{
				if (m_nCondSectionDepth != 0)
				{
					SetLastError(XML_UnclosedConditionalSection);
					return false;
				}
				// append the rest to internal subset value but not the closing ']'
				if (m_pInternalSubsetValueSb->GetLength() > 0)
				{
					BFX_ASSERT(m_nCurPos > 0 && m_pChars[m_nCurPos-1] == ']');
					SaveParsingBuffer(m_nCurPos - 1);
					m_strInternalSubsetValue = m_pInternalSubsetValueSb->ToString();
					m_pInternalSubsetValueSb->SetLength(0);
				}
				// check '>'
				if (GetToken(false) != TokenGreaterThan)
				{
					SetLastError(XML_UnexpectedToken, ">");
					return false;
				}
			}
			else
			{
				SetLastError(XML_ExpectDtdMarkup);
				return false;
			}
			return true;
		case TokenEof:
			if (IsParsingInternalSubset() && !m_bFreeFloatingDtd)
			{
				SetLastError(XML_IncompleteDtdContent);
				return false;
			}
			if (m_nCondSectionDepth != 0)
			{
				SetLastError(XML_UnclosedConditionalSection);
				return false;
			}
			return true;
		case TokenError:
			return false;
		default:
			BFX_ASSERT(false);
			break;
		}
		if (IsErrorOccurred())
			return false;

		BFX_ASSERT(m_scanningFunction == ScanningFunctionSubsetContent);

		if (m_nCurEntityId != nStartTagEntityId)
		{
			if (m_bValidate)
			{
				// TODO: validate
				BFX_ASSERT(false);
			}
			else if (!m_bV1Compat)
			{
				SetLastError(XML_Sch_ParEntityRefNesting);
				return false;
			}
		}
	}
	return true;
}

bool REFXMLDtdParser::ParseAttlistDecl()
{
	REF<REFXMLSchemaElementDecl> pElementDecl;
	REF<REFXMLQualifiedName> pElementName;
	REF<REFXMLSchemaAttDef> pAttrDef;
	REF<REFXMLQualifiedName> pAttrName;

	switch (GetToken(true))
	{
	case TokenQName:
		break;
	default:
		goto UnexpectedError;
	}

	if ((pElementName = GetNameQualified(true)) == NULL)
		return false;

	m_pSchemaInfo->GetElementDecls()->Lookup(pElementName, pElementDecl);
	if (pElementDecl == NULL)
	{
		m_pSchemaInfo->GetUndeclaredElementDecls()->Lookup(pElementName, pElementDecl);
		if (pElementDecl == NULL)
		{
			pElementDecl = new REFXMLSchemaElementDecl(pElementName, pElementName->GetNamespace(), REFXMLSchemaDecl::SchemaTypeDTD);
			m_pSchemaInfo->GetUndeclaredElementDecls()->Set(pElementName, pElementDecl);
		}
	}

	for (;;)
	{
		switch (GetToken(false))
		{
		case TokenQName:
			pAttrName = GetNameQualified(true);
			if (pAttrName == NULL)
				return false;

			pAttrDef = new REFXMLSchemaAttDef(pAttrName, pAttrName->GetNamespace());
			pAttrDef->SetDeclaredInExternal(!IsParsingInternalSubset());
			// TODO: lineNo and linePos
			break;
		case TokenGreaterThan:
			if (m_bV1Compat)
			{
				// check xml:space and xml:lang
				if (pAttrDef != NULL && pAttrDef->GetPrefix().GetLength() > 0 &&
					pAttrDef->GetPrefix() == USTR("xml") && pAttrDef->GetName()->GetName() == USTR("space"))
				{
					pAttrDef->SetReserved(REFXMLSchemaAttDef::ReserveXmlSpace);
					if (pAttrDef->GetDataType() != XMLTokenizedType_ENUMERATION)
					{
						SetLastError(XML_EnumerationRequired);
						return false;
					}
					// NOTE: We can check XMLSpace here.
				}
			}
			return true;
		default:
			goto UnexpectedError;
		}

		if (!ParseAttlistType(pAttrDef, pElementDecl))
			return false;
		if (!ParseAttlistDefault(pAttrDef))
			return false;

		// check xml:space and xml:lang
		if (pAttrDef->GetPrefix().GetLength() > 0 && pAttrDef->GetPrefix() == USTR("xml"))
		{
			if (pAttrDef->GetName()->GetName() == USTR("space"))
			{
				if (m_bV1Compat)
				{
					UniString val = pAttrDef->GetDefaultValueExpanded().Trim();
					if (val == USTR("preserve") || val == USTR("default"))
					{
						pAttrDef->SetReserved(REFXMLSchemaAttDef::ReserveXmlSpace);
					}
				}
				else
				{
					pAttrDef->SetReserved(REFXMLSchemaAttDef::ReserveXmlSpace);
					if (pAttrDef->GetDataType() != XMLTokenizedType_ENUMERATION)
					{
						SetLastError(XML_EnumerationRequired);
					}
					// NOTE: We can check XMLSpace here.
				}
			}
			else if (pAttrDef->GetName()->GetName() == USTR("lang"))
			{
				pAttrDef->SetReserved(REFXMLSchemaAttDef::ReserveXmlLang);
			}
		}
		// add attribute to element decl
		if (pElementDecl->GetAttDef(pAttrDef->GetName()) == NULL)
		{
			pElementDecl->AddAttDef(pAttrDef);
		}
	}

UnexpectedError:
	if (!IsErrorOccurred())
	{
		BFX_ASSERT1(false, "This is an unexpected error that should have been handled in the ScanXXX methods.");
		SetLastError(XML_InternalError);
	}
	return false;
}

bool REFXMLDtdParser::ParseAttlistType(REFXMLSchemaAttDef* pAttrDef, REFXMLSchemaElementDecl* pElementDecl)
{
	Token token = GetToken(true);
	if (token == TokenError)
		return false;

	if (token != TokenCDATA)
	{
		pElementDecl->SetHasNonCDataAttribute(true);
	}

	if (IsAttributeValueType(token))
	{
		pAttrDef->SetDataType((XMLTokenizedType)token);

		switch (token)
		{
		case TokenNOTATION:
			break;
		case TokenID:
			if (m_bValidate && pElementDecl->IsIdDeclared())
			{
				REFXMLSchemaAttDef* pIdAttrDef = pElementDecl->GetAttDef(pAttrDef->GetName());
				if (pIdAttrDef == NULL || pIdAttrDef->GetDataType() != XMLTokenizedType_ID)
				{
					// TODO: validate
					BFX_ASSERT(false);
				}
			}
			pElementDecl->SetIdDeclared(true);
			return true;
		default:
			return true;
		}

		// check notation constrains
		if (m_bValidate)
		{
			// TODO: validate
			BFX_ASSERT(false);
		}

		if (GetToken(true) != TokenLeftParen)
		{
			goto UnexpectedError;
		}

		// parse notation list
		if (GetToken( false ) != TokenName)
		{
			goto UnexpectedError;
		}

		for (;;)
		{
			UniString strNotationName = GetNameString();
			if (!m_pSchemaInfo->GetNotations()->Contains(strNotationName))
			{
				// NOTE: save undeclared notation, we may validate it after parsing.
				if (m_pUndeclaredNotations == NULL)
				{
					m_pUndeclaredNotations = new UndeclaredNotationMap();
				}
				REF<UndeclaredNotation> pun = new UndeclaredNotation(strNotationName, GetLineNo(), GetLinePos() - strNotationName.GetLength());
				REF<UndeclaredNotation> ploggedUn;
				m_pUndeclaredNotations->Lookup(strNotationName, ploggedUn);
				pun->m_pNext = ploggedUn;
				m_pUndeclaredNotations->Set(strNotationName, pun);
			}
			
			if (m_bValidate && !m_bV1Compat && pAttrDef->GetValues()->Contains(strNotationName))
			{
				// TODO: validate
				BFX_ASSERT(false);
			}
			pAttrDef->GetValues()->Add(strNotationName);

			switch (GetToken(false))
			{
			case TokenOr:
				if (GetToken(false) != TokenName)
				{
					goto UnexpectedError;
				}
				continue;
			case TokenRightParen:
				return true;
			default:
				goto UnexpectedError;
			}
		}

	}
	else if (token == TokenLeftParen)
	{
		pAttrDef->SetDataType(XMLTokenizedType_ENUMERATION);

		// parse nmtoken list
		token = GetToken(false);
		if (token != TokenNmtoken ) 
			goto UnexpectedError;
		pAttrDef->GetValues()->Add(GetNameString());

		UniString strNmtoken;
		for (;;)
		{
			switch (GetToken(false))
			{
			case TokenOr:
				if (GetToken(false) != TokenNmtoken)
					goto UnexpectedError;
				strNmtoken = GetNmtokenString();
				if (m_bValidate && !m_bV1Compat && pAttrDef->GetValues()->Contains(strNmtoken))
				{
					// TODO: validate
					BFX_ASSERT(false);
				}
				pAttrDef->GetValues()->Add(strNmtoken);
				break;
			case TokenRightParen:
				return true;
			default:
				goto UnexpectedError;
			}
		}
	}
	else
	{
		goto UnexpectedError;
	}

UnexpectedError:
	if (!IsErrorOccurred())
	{
		BFX_ASSERT1(false, "This is an unexpected error that should have been handled in the ScanXXX methods.");
		SetLastError(XML_InternalError);
	}
	return false;
}

bool REFXMLDtdParser::ParseAttlistDefault(REFXMLSchemaAttDef* pAttrDef)
{
	switch (GetToken(true))
	{
	case TokenREQUIRED:
		pAttrDef->SetPresence(REFXMLSchemaDecl::UseRequired);
		return true;
	case TokenIMPLIED:
		pAttrDef->SetPresence(REFXMLSchemaDecl::UseImplied);
		return true;
	case TokenFIXED:
		pAttrDef->SetPresence(REFXMLSchemaDecl::UseFixed);
		if (GetToken(true) != TokenLiteral)
		{
			goto UnexpectedError;
		}
		break;
	case TokenLiteral:
		break;
	default:
		goto UnexpectedError;
	}

	if (m_bValidate && pAttrDef->GetDataType() == XMLTokenizedType_ID)
	{
		// TODO: validate
		BFX_ASSERT(false);
	}
	if (pAttrDef->GetDataType() != XMLTokenizedType_CDATA)
	{
		// non-CDATA attribute type normalization - strip spaces
		pAttrDef->SetDefaultValueExpanded(GetValueWithStrippedSpaces());
	}
	else
	{
		pAttrDef->SetDefaultValueExpanded(GetValue());
	}
	// TODO: set lineNo, linePos of pAttrDef

	return true;

UnexpectedError:
	if (!IsErrorOccurred())
	{
		BFX_ASSERT1(false, "This is an unexpected error that should have been handled in the ScanXXX methods.");
		SetLastError(XML_InternalError);
	}
	return false;
}

bool REFXMLDtdParser::ParseElementDecl()
{
	// element name
	Token token = GetToken(true);
	if (token == TokenError)
		return false;
	BFX_ASSERT1((token == TokenQName), "This is an unexpected error that should have been handled in the ScanXXX methods.");

	// get schema decl for element
	REF<REFXMLSchemaElementDecl> pElementDecl;
	REFXMLQualifiedName* pName = GetNameQualified(true);
	if (pName == NULL)
		return false;
	m_pSchemaInfo->GetElementDecls()->Lookup(pName, pElementDecl);
	if (pElementDecl != NULL)
	{
		if (m_bValidate)
		{
			// TODO: validate
			BFX_ASSERT(false);
		}
	}
	else
	{
		if (m_pSchemaInfo->GetUndeclaredElementDecls()->Lookup(pName, pElementDecl))
		{
			m_pSchemaInfo->GetUndeclaredElementDecls()->Remove(pName);
		}
		else
		{
			pElementDecl = new REFXMLSchemaElementDecl(pName, pName->GetNamespace(), REFXMLSchemaDecl::SchemaTypeDTD);
		}
		m_pSchemaInfo->GetElementDecls()->Set(pName, pElementDecl);
	}
	pElementDecl->SetDeclaredInExternal(!IsParsingInternalSubset());

	// content spec
	int nStartParenEntityId = m_nCurEntityId;
	switch (GetToken(true))
	{
	case TokenEMPTY:
		pElementDecl->SetContentValidator(REFXMLContentValidator::Empty());
		goto End;
	case TokenANY:
		pElementDecl->SetContentValidator(REFXMLContentValidator::Any());
		goto End;
	case TokenLeftParen:
		switch (GetToken(false))
		{
		case TokenPCDATA:
			{
				REF<REFXMLParticleContentValidator> pcv = new REFXMLParticleContentValidator(XMLSchemaContentType_Mixed);
				pcv->Start();
				pcv->OpenGroup();

				ParseElementMixedContent(pcv, nStartParenEntityId);

				pElementDecl->SetContentValidator(pcv->Finish());
				goto End;
			}
		case TokenNone:
			{
				REF<REFXMLParticleContentValidator> pcv = new REFXMLParticleContentValidator(XMLSchemaContentType_ElementOnly);
				pcv->Start();
				pcv->OpenGroup();

				ParseElementOnlyContent(pcv, nStartParenEntityId);

				pElementDecl->SetContentValidator(pcv->Finish());
				goto End;
			}
		default:
			goto UnexpectedError;
		}
		/* no break */
	default:
		goto UnexpectedError;
	}
End:
	if (GetToken(false) != TokenGreaterThan)
	{
		SetLastError(XML_UnexpectedToken, ">");
		return false;
	}
	return true;

UnexpectedError:
	if (!IsErrorOccurred())
	{
		BFX_ASSERT1(false, "This is an unexpected error that should have been handled in the ScanXXX methods.");
		SetLastError(XML_InternalError);
	}
	return false;
}

bool REFXMLDtdParser::ParseEntityDecl()
{
	bool bIsParamEntity = false;
	REF<REFXMLSchemaEntity> pEntity;
	UniString strSystemId;
	UniString strPublicId;

	// get entity name and type
	Token token;
	REF<REFXMLQualifiedName> pEntityName;
	switch (GetToken(true))
	{
	case TokenPercent:
		bIsParamEntity = true;
		if ((token = GetToken(true)) != TokenName)
		{
			if (token == TokenError)
				return false;
			goto UnexpectedError;
		}
		/* no break */
	case TokenName:
		// create entity object
		if ((pEntityName = GetNameQualified(false)) == NULL)
			return false;
		pEntity = new REFXMLSchemaEntity(pEntityName, bIsParamEntity);

		pEntity->SetBaseURI(GetBaseURI());
		pEntity->SetDeclaredURI(m_strExternalDtdBaseUri.IsEmpty() ? m_strDocumentBaseUri : m_strExternalDtdBaseUri);

		if (bIsParamEntity)
		{
			if (!m_pSchemaInfo->GetParameterEntities()->Contains(pEntityName))
			{
				m_pSchemaInfo->GetParameterEntities()->Set(pEntityName, pEntity);
			}
		}
		else
		{
			if (!m_pSchemaInfo->GetParameterEntities()->Contains(pEntityName))
			{
				m_pSchemaInfo->GetGeneralEntities()->Set(pEntityName, pEntity);
			}
		}
		pEntity->SetDeclaredInExternal(!IsParsingInternalSubset());
		pEntity->SetProcessed(true);
		break;
	default:
		goto UnexpectedError;
	}

	token = GetToken( true );
	switch (token)
	{
	case TokenPUBLIC:
	case TokenSYSTEM:
		if (!ParseExternalId(token, TokenEntityDecl, OUT strPublicId, OUT strSystemId))
			return false;
		pEntity->SetExternal(true);
		pEntity->SetUrl(strSystemId);
		pEntity->SetPubid(strPublicId);

		if ((token = GetToken(false)) == TokenNData)
		{
			if (bIsParamEntity)
			{
				SetLastError(XML_UnexpectedToken, ">", m_nCurPos - 5);	// 5 == strlen("NDATA")
				return false;
			}
			if (!m_bWhitespaceBeforeToken)
			{
				SetLastError(XML_ExpectingWhiteSpace, "NDATA", m_nCurPos - 5);
				return false;
			}

			if ((token = GetToken(true)) != TokenName)
			{
				if (token == TokenError)
					return false;
				goto UnexpectedError;
			}
			pEntity->SetNData(GetNameQualified(false));
			if (pEntity->GetNData() == NULL)
				return false;
			UniString strNotationName = pEntity->GetNData()->GetName();
			if (!m_pSchemaInfo->GetNotations()->Contains(strNotationName))
			{
				// NOTE: save undeclared notation, we may validate it after parsing.
				if (m_pUndeclaredNotations == NULL)
				{
					m_pUndeclaredNotations = new UndeclaredNotationMap();
				}
				REF<UndeclaredNotation> pun = new UndeclaredNotation(strNotationName, GetLineNo(), GetLinePos() - strNotationName.GetLength());
				REF<UndeclaredNotation> ploggedUn;
				m_pUndeclaredNotations->Lookup(strNotationName, ploggedUn);
				pun->m_pNext = ploggedUn;
				m_pUndeclaredNotations->Set(strNotationName, pun);
			}
		}
		else if (token == TokenError)
			return false;
		break;
	case TokenLiteral:
		pEntity->SetText(GetValue());
		// TODO: lineNo and linePos
		break;
	default:
		goto UnexpectedError;
	}

	if ((token = GetToken(false)) == TokenGreaterThan)
	{
		pEntity->SetProcessed(false);
		return true;
	}
	else if (token == TokenError)
		return false;

UnexpectedError:
	if (!IsErrorOccurred())
	{
		BFX_ASSERT1(false, "This is an unexpected error that should have been handled in the ScanXXX methods.");
		SetLastError(XML_InternalError);
	}
	return false;
}
bool REFXMLDtdParser::ParseNotationDecl()
{
	Token token;
	REF<REFXMLSchemaNotation> pNotation;
	REF<REFXMLQualifiedName> pNotationName;

	// notation name
	if ((token = GetToken(true)) != TokenName)
	{
		goto UnexpectedError;
	}

	pNotationName = GetNameQualified(false);
	if (!m_pSchemaInfo->GetNotations()->Contains(pNotationName->GetName()))
	{
		if (m_pUndeclaredNotations != NULL)
		{
			m_pUndeclaredNotations->Remove(pNotationName->GetName());
		}
		pNotation = new REFXMLSchemaNotation(pNotationName);
		m_pSchemaInfo->GetNotations()->Set(pNotation->GetName()->GetName(), pNotation);
	}
	else
	{
		// duplicate notation
		if (m_bValidate)
		{
			// TODO: validate
			BFX_ASSERT(false);
		}
	}

	// public / system id
	token = GetToken(true);
	if (token == TokenSYSTEM || token == TokenPUBLIC )
	{
		UniString strNotationPublicId, strNotationSystemId;
		ParseExternalId(token, TokenNOTATION, OUT strNotationPublicId, OUT strNotationSystemId);
		if (pNotation != NULL)
		{
			pNotation->SetSystemLiteral(strNotationSystemId);
			pNotation->SetPubid(strNotationPublicId);
		}
	}
	else
	{
		goto UnexpectedError;
	}

	if ((token = GetToken(false)) == TokenGreaterThan)
	{
		return true;
	}

UnexpectedError:
	if (!IsErrorOccurred())
	{
		BFX_ASSERT1(false, "This is an unexpected error that should have been handled in the ScanXXX methods.");
		SetLastError(XML_InternalError);
	}
	return false;
}

bool REFXMLDtdParser::ParseComment()
{
	bool bResult;

	SaveParsingBuffer();
	if (IsSaveInternalSubsetValue())
	{
		bResult = m_pReaderAdapter->ParseComment(m_pInternalSubsetValueSb);
		m_pInternalSubsetValueSb->Append(USTR("-->"));
	}
	else
	{
		bResult = m_pReaderAdapter->ParseComment(NULL);
	}
	LoadParsingBuffer();
	return bResult;
}

bool REFXMLDtdParser::ParsePI()
{
	bool bResult;

	SaveParsingBuffer();
	if (IsSaveInternalSubsetValue())
	{
		bResult = m_pReaderAdapter->ParsePI(m_pInternalSubsetValueSb);
		m_pInternalSubsetValueSb->Append(USTR("?>"));
	}
	else
	{
		bResult = m_pReaderAdapter->ParsePI(NULL);
	}
	LoadParsingBuffer();
	return bResult;
}

bool REFXMLDtdParser::ParseCondSection()
{
	//int nCsEntityId = m_nCurEntityId;

	switch (GetToken(false))
	{
	case TokenINCLUDE:
		if (GetToken(false) != TokenRightBracket)
		{
			goto UnexpectedError;
		}
		if (m_bValidate)
		{
			// TODO: validate
			BFX_ASSERT(false);
		}
		m_nCondSectionDepth ++;
		break;
	case TokenIGNORE:
		if (GetToken(false) != TokenRightBracket)
		{
			goto UnexpectedError;
		}
		if (m_bValidate)
		{
			// TODO: validate
			BFX_ASSERT(false);
		}
		// the content of the ignore section is parsed & skipped by scanning function
		if (GetToken(false) != TokenCondSectionEnd)
		{
			goto UnexpectedError;
		}
		if (m_bValidate)
		{
			// TODO: validate
			BFX_ASSERT(false);
		}
		break;
	default:
UnexpectedError:
		if (!IsErrorOccurred())
		{
			BFX_ASSERT1(false, "This is an unexpected error that should have been handled in the ScanXXX methods.");
			SetLastError(XML_InternalError);
		}
		return false;
	}

	return true;
}

bool REFXMLDtdParser::ParseElementOnlyContent(REFXMLParticleContentValidator* pcv, int nStartParenEntityId)
{
	Token parsingSchema = TokenNone;
	// int nConnectorEntityId = nStartParenEntityId;
	// int nContentEntityId = -1;
	REF<REFXMLQualifiedName> pName;
	for (;;)
	{
		switch (GetToken(false))
		{
		case TokenQName:
			pName = GetNameQualified(true);
			if (pName == NULL)
				return false;
			pcv->AddName(pName);
			if (m_bValidate)
			{
				// TODO: validate
				BFX_ASSERT(false);
			}
			if (!ParseHowMany(pcv))
				goto UnexpectedError;
			break;
		case TokenLeftParen:
			pcv->OpenGroup();
			if (m_bValidate)
			{
				// TODO: validate
				BFX_ASSERT(false);
			}
			ParseElementOnlyContent(pcv, m_nCurEntityId);
			break;
		case TokenGreaterThan:
			SetLastError(XML_InvalidContentModel);
			return false;
		default:
			goto UnexpectedError;
		}
		if (IsErrorOccurred())
			return false;

		switch (GetToken(false))
		{
		case TokenComma:
			if (parsingSchema == TokenOr)
			{
				SetLastError(XML_InvalidContentModel);
				return false;
			}
			pcv->AddSequence();
			parsingSchema = TokenComma;
			break;
		case TokenOr:
			if (parsingSchema == TokenComma)
			{
				SetLastError(XML_InvalidContentModel);
				return false;
			}
			pcv->AddChoice();
			parsingSchema = TokenOr;
			break;
		case TokenRightParen:
			pcv->CloseGroup();
			if (m_bValidate && m_nCurEntityId != nStartParenEntityId)
			{
				// TODO: validate
				BFX_ASSERT(false);
			}
			return ParseHowMany(pcv);
		case TokenGreaterThan:
			SetLastError(XML_InvalidContentModel);
			return false;
		default:
			goto UnexpectedError;
		}
	}

UnexpectedError:
	if (!IsErrorOccurred())
	{
		BFX_ASSERT1(false, "This is an unexpected error that should have been handled in the ScanXXX methods.");
		SetLastError(XML_InternalError);
	}
	return false;
}

bool REFXMLDtdParser::ParseHowMany(REFXMLParticleContentValidator* pcv)
{
	switch (GetToken(false))
	{
	case TokenStar:
		pcv->AddStar();
		return true;
	case TokenQMark:
		pcv->AddQMark();
		return true;
	case TokenPlus:
		pcv->AddPlus();
		return true;
	case TokenError:
		return false;
	default:
		return true;
	}
}

bool REFXMLDtdParser::ParseExternalId(Token idTokenType, Token declType, OUT UniString& strPublicId, OUT UniString& strSystemId)
{
	XMLLineInfo keywordLineInfo(GetLineNo(), GetLinePos() - 6);
	strPublicId.SetEmpty();
	strSystemId.SetEmpty();

	Token token = GetToken(true);
	if (token != TokenLiteral)
	{
		// NOTE: in case of TokenError, the last error already been set.
		if (token != TokenError)
			SetLastError(XML_UnexpectedToken, "\",'");
		return false;
	}

	if (idTokenType == TokenSYSTEM)
	{
		strSystemId = GetValue();

		if (strSystemId.IndexOf('#') >= 0)
		{
			// "Fragment identifier '#' cannot be part of the system identifier : strSystemId;
			SetLastError(XML_InvalidCharacter, '#');
			return false;
		}

		if (declType == TokenDOCTYPE && !m_bFreeFloatingDtd)
		{
			m_literalLineInfo.m_nLinePos ++;
			m_pReaderAdapter->OnSystemId(strSystemId, keywordLineInfo, m_literalLineInfo);
		}
	}
	else
	{
		BFX_ASSERT(idTokenType == TokenPUBLIC);
		strPublicId = GetValue();

		// verify if it contains chars valid for public ids
		int i;
		for (i = 0; i < strPublicId.GetLength(); i ++)
		{
			if (!XMLCharType::IsPubidChar(strPublicId[i]))
				break;
		}
		if (i != strPublicId.GetLength())
		{
			SetLastError(XML_InvalidCharacter, m_nCurPos - 1 - strPublicId.GetLength() + i, (char)strPublicId[i]);
			return false;
		}

		if (declType == TokenDOCTYPE && !m_bFreeFloatingDtd)
		{
			m_literalLineInfo.m_nLinePos ++;
			m_pReaderAdapter->OnPublicId(strPublicId, keywordLineInfo, m_literalLineInfo);

			token = GetToken(false);
			if (token == TokenLiteral)
			{
				if (!m_bWhitespaceBeforeToken)
				{
					SetLastError(XML_ExpectingWhiteSpace/*, m_literalLineInfo.m_nLineNo, m_literalLineInfo.m_nLinePos*/);
					return false;
				}
				strSystemId = GetValue();
				m_literalLineInfo.m_nLinePos ++;
				m_pReaderAdapter->OnSystemId(strSystemId, keywordLineInfo, m_literalLineInfo);
			}
			else
			{
				if (token != TokenError)
					SetLastError(XML_UnexpectedToken, "\",'");
				return false;
			}
		}
		else
		{
			token = GetToken(false);
			if (token == TokenError)
			{
				return false;
			}
			else if (token == TokenLiteral)
			{
				if (!m_bWhitespaceBeforeToken)
				{
					SetLastError(XML_ExpectingWhiteSpace/*, m_literalLineInfo.m_nLineNo, m_literalLineInfo.m_nLinePos*/);
					return false;
				}
				strSystemId = GetValue();
			}
			else if (declType != TokenNOTATION)
			{
				SetLastError(XML_UnexpectedToken, "\",'");
				return false;
			}
		}
	}
	return true;
}

UniString REFXMLDtdParser::ParseUnexpectedToken(int nStartPos)
{
	if (XMLCharType::IsNCNameChar(m_pChars[nStartPos]))
	{
		int nEndPos = nStartPos + 1;
		while (XMLCharType::IsNCNameChar(m_pChars[nEndPos]))
		{
			nEndPos++;
		}
		return UniString(m_pChars + nStartPos, nEndPos - nStartPos);
	}
	else
	{
		BFX_ASSERT(nStartPos < m_nCharsUsed);
		return UniString(m_pChars + nStartPos, 1);
	}
}

bool REFXMLDtdParser::ParseElementMixedContent(REFXMLParticleContentValidator* pcv, int nStartParenEntityId)
{
	bool bHasNames = false;
	//int nConnectorEntityId = -1;
	//int nContentEntityId = m_nCurEntityId;
	REFXMLQualifiedName* pName;

	for (;;)
	{
		switch (GetToken(false))
		{
		case TokenRightParen:
			// TODO: validate...
			if (GetToken( false ) == TokenStar && bHasNames)
			{
				pcv->AddStar();
			}
			else if (bHasNames)
			{
				SetLastError(XML_UnexpectedToken, '*');
				return false;
			}
			return true;
		case TokenOr:
			if (!bHasNames)
			{
				bHasNames = true;
			}
			else
			{
				pcv->AddChoice();
			}
			if (m_bValidate)
			{
				// TODO: validate...
				BFX_ASSERT(false);
			}

			if (GetToken(false) != TokenQName)
			{
				goto UnexpectedError;
			}

			pName = GetNameQualified(true);
			if (pName == NULL)
				return false;
			if (m_bValidate  && pcv->Exists(pName))
			{
				// TODO: validate
				BFX_ASSERT(false);
			}
			pcv->AddName(pName);

			continue;
		default:
UnexpectedError:
			if (!IsErrorOccurred())
			{
				BFX_ASSERT1(false, "This is an unexpected error that should have been handled in the ScanXXX methods.");
				SetLastError(XML_InternalError);
			}
			return false;
		}
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return false;
}

int REFXMLDtdParser::ReadData()
{
	SaveParsingBuffer();
	int nCharsRead = m_pReaderAdapter->ReadData();
	LoadParsingBuffer();
	return nCharsRead;
}

bool REFXMLDtdParser::ReadDataInName()
{
	int iOffset = m_nCurPos - m_nTokenStartPos;
	m_nCurPos = m_nTokenStartPos;
	bool bNewDataRead = (ReadData() != 0);
	m_nTokenStartPos = m_nCurPos;
	m_nCurPos += iOffset;
	return bNewDataRead;
}

void REFXMLDtdParser::SaveParsingBuffer(int nInternalSubsetValueEndPos)
{
	if (IsSaveInternalSubsetValue())
	{
		BFX_ASSERT(m_pInternalSubsetValueSb != NULL);

		int nReaderCurPos = m_pReaderAdapter->GetCurrentPosition();
		if (nInternalSubsetValueEndPos - nReaderCurPos > 0)
		{
			m_pInternalSubsetValueSb->Append(m_pChars, nReaderCurPos, nInternalSubsetValueEndPos - nReaderCurPos);
		}
	}
	m_pReaderAdapter->SetCurrentPosition(m_nCurPos);
}

REFXMLDtdParser::Token REFXMLDtdParser::GetToken(bool bNeedWhiteSpace)
{
	m_bWhitespaceBeforeToken = false;
	for (;;)
	{
		switch (m_pChars[m_nCurPos])
		{
		case 0:		// \0
			if (m_nCurPos == m_nCharsUsed)
			{
				goto ReadData;
			}
			else
			{
				SetLastError(XML_InvalidCharacter, m_pChars[m_nCurPos]);
				return TokenError;
			}
			break;
		case 0xA:
			m_bWhitespaceBeforeToken = true;
			m_nCurPos++;
			m_pReaderAdapter->OnNewLine(m_nCurPos);
			continue;
		case 0xD:
			m_bWhitespaceBeforeToken = true;
			if (m_pChars[m_nCurPos+1] == 0xA)
			{
				if (m_bNormalize)
				{
					SaveParsingBuffer();	// EOL normalization of 0xD 0xA
					m_pReaderAdapter->SetCurrentPosition(m_pReaderAdapter->GetCurrentPosition() + 1);
				}
				m_nCurPos += 2;
			}
			else if (m_nCurPos+1 < m_nCharsUsed || m_pReaderAdapter->IsEof())
			{
				m_pChars[m_nCurPos] = 0xA;	// EOL normalization of 0xD
				m_nCurPos++;
			}
			else
			{
				goto ReadData;
			}
			m_pReaderAdapter->OnNewLine(m_nCurPos);
			continue;
		case 0x9:
		case 0x20:
			m_bWhitespaceBeforeToken = true;
			m_nCurPos++;
			continue;
		case '%':
			if (m_nCharsUsed - m_nCurPos < 2)
			{
				goto ReadData;
			}
			if (!XMLCharType::IsWhiteSpace(m_pChars[m_nCurPos+1]))
			{
				// Ignore entity references.
				if (m_scanningFunction == ScanningFunctionCondSection3)
				{
					m_nCurPos++;
				}
				else
				{
					HandleEntityReference(true, false, false);
				}
				continue;
			}
			/* no break */
		default:
			if (bNeedWhiteSpace && !m_bWhitespaceBeforeToken && m_scanningFunction != ScanningFunctionParamEntitySpace)
			{
				SetLastError(XML_ExpectingWhiteSpace, CSTR(ParseUnexpectedToken(m_nCurPos)));
				return TokenError;
			}
			m_nTokenStartPos = m_nCurPos;
SwitchAgain:

			switch (m_scanningFunction)
			{
			case ScanningFunctionName:
				return ScanNameExpected();
			case ScanningFunctionQName:
				return ScanQNameExpected();
			case ScanningFunctionNmtoken:
				return ScanNmtokenExpected();
			case ScanningFunctionSubsetContent:
				return ScanSubsetContent();
			case ScanningFunctionDoctype1:
				return ScanDoctype1();
			case ScanningFunctionDoctype2:
				return ScanDoctype2();
			case ScanningFunctionElement1:
				return ScanElement1();
			case ScanningFunctionElement2:
				return ScanElement2();
			case ScanningFunctionElement3:
				return ScanElement3();
			case ScanningFunctionElement4:
				return ScanElement4();
			case ScanningFunctionElement5:
				return ScanElement5();
			case ScanningFunctionElement6:
				return ScanElement6();
			case ScanningFunctionElement7:
				return ScanElement7();
			case ScanningFunctionAttlist1:
				return ScanAttlist1();
			case ScanningFunctionAttlist2:
				return ScanAttlist2();
			case ScanningFunctionAttlist3:
				return ScanAttlist3();
			case ScanningFunctionAttlist4:
				return ScanAttlist4();
			case ScanningFunctionAttlist5:
				return ScanAttlist5();
			case ScanningFunctionAttlist6:
				return ScanAttlist6();
			case ScanningFunctionAttlist7:
				return ScanAttlist7();
			case ScanningFunctionNotation1:
				return ScanNotation1();
			case ScanningFunctionSystemId:
				return ScanSystemId();
			case ScanningFunctionPublicId1:
				return ScanPublicId1();
			case ScanningFunctionPublicId2:
				return ScanPublicId2();
			case ScanningFunctionEntity1:
				return ScanEntity1();
			case ScanningFunctionEntity2:
				return ScanEntity2();
			case ScanningFunctionEntity3:
				return ScanEntity3();
			case ScanningFunctionCondSection1:
				return ScanCondSection1();
			case ScanningFunctionCondSection2:
				return ScanCondSection2();
			case ScanningFunctionCondSection3:
				return ScanCondSection3();
			case ScanningFunctionClosingTag:
				return ScanClosingTag();
			case ScanningFunctionParamEntitySpace:
				m_bWhitespaceBeforeToken = true;
				m_scanningFunction = m_savedScanningFunction;
				goto SwitchAgain;
			default:
				BFX_ASSERT(false);
				return TokenNone;
			}
		}
ReadData:
		if (m_pReaderAdapter->IsEof() || ReadData() == 0)
		{
			if (HandleEntityEnd(false))
			{
				continue;
			}
			if (m_scanningFunction == ScanningFunctionSubsetContent)
			{
				return TokenEof;
			}
			else
			{
				SetLastError(XML_IncompleteDtdContent);
				return TokenError;
			}
		}
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return TokenError;
}

bool REFXMLDtdParser::ScanQName(bool bIsQName)
{
	m_nTokenStartPos = m_nCurPos;
	int nColonOffset = -1;

	for (;;)
	{
		if (!XMLCharType::IsStartNameChar(m_pChars[m_nCurPos]) && m_pChars[m_nCurPos] != ':')
		{
			if (m_nCurPos == m_nCharsUsed)
			{
				// end of buffer
				if (ReadDataInName())
					continue;
				SetLastError(XML_UnexpectedEOF, "Name");
				return false;
			}
			else if (m_pChars[m_nCurPos] != ':' || m_bSupportNamespaces)
			{
				SetLastError(XML_BadStartNameChar, m_pChars[m_nCurPos]);
				return false;
			}
		}
		m_nCurPos++;

ContinueName:
		while (XMLCharType::IsNCNameChar(m_pChars[m_nCurPos]))
		{
			m_nCurPos++;
		}

		if (m_pChars[m_nCurPos] == ':')
		{
			if (bIsQName)
			{
				if (nColonOffset != -1)
				{
					SetLastError(XML_BadNameChar, ':');
					return false;
				}
				nColonOffset = m_nCurPos - m_nTokenStartPos;
				m_nCurPos++;
				continue;
			}
			else
			{
				m_nCurPos++;
				goto ContinueName;
			}
		}
		else if (m_nCharsUsed - m_nCurPos == 0)
		{
			// end of buffer
			if (ReadDataInName())
			{
				goto ContinueName;
			}
			if (m_nTokenStartPos == m_nCurPos)
			{
				SetLastError(XML_UnexpectedEOF, "Name");
				return false;
			}
		}
		// end of name
		m_nColonPos = (nColonOffset == -1 ) ? -1 : m_nTokenStartPos + nColonOffset;
		return true;
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return false;
}

bool REFXMLDtdParser::ScanNmtoken()
{
	m_nTokenStartPos = m_nCurPos;

	for (;;)
	{
		while (XMLCharType::IsNCNameChar(m_pChars[m_nCurPos]) || m_pChars[m_nCurPos] == ':')
		{
			m_nCurPos++;
		}

		if (m_pChars[m_nCurPos] != 0)
		{
			if (m_nCurPos - m_nTokenStartPos == 0)
			{
				SetLastError(XML_BadNameChar, m_pChars[m_nCurPos]);
				return false;
			}
			return true;
		}

		int nLen = m_nCurPos - m_nTokenStartPos;
		m_nCurPos = m_nTokenStartPos;
		if (ReadData() == 0)
		{
			if (nLen > 0)
			{
				m_nTokenStartPos = m_nCurPos;
				m_nCurPos += nLen;
				return true;
			}
			SetLastError(XML_UnexpectedEOF, "NmToken");
			return false;
		}

		m_nTokenStartPos = m_nCurPos;
		m_nCurPos += nLen;
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return false;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanDoctype1()
{
	switch (m_pChars[m_nCurPos])
	{
	case 'P':
		if (!EatPublicKeyword())
		{
			SetLastError(XML_ExpectExternalOrClose);
			return TokenError;
		}
		m_nextScaningFunction = ScanningFunctionDoctype2;
		m_scanningFunction = ScanningFunctionPublicId1;
		return TokenPUBLIC;
	case 'S':
		if (!EatSystemKeyword())
		{
			SetLastError(XML_ExpectExternalOrClose);
			return TokenError;
		}
		m_nextScaningFunction = ScanningFunctionDoctype2;
		m_scanningFunction = ScanningFunctionSystemId;
		return TokenSYSTEM;
	case '[':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionSubsetContent;
		return TokenLeftBracket;
	case '>':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionSubsetContent;
		return TokenGreaterThan;
	default:
		SetLastError(XML_ExpectExternalOrClose);
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanDoctype2()
{
	switch (m_pChars[m_nCurPos])
	{
	case '[':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionSubsetContent;
		return TokenLeftBracket;
	case '>':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionSubsetContent;
		return TokenGreaterThan;
	default:
		SetLastError(XML_ExpectSubOrClose);
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanSubsetContent()
{
	for (;;)
	{
		switch (m_pChars[m_nCurPos])
		{
		case '<':
			switch (m_pChars[m_nCurPos+1])
			{
			case '!':
				switch (m_pChars[m_nCurPos+2])
				{
				case 'E':
					// <!ELEMENT
					if (m_pChars[m_nCurPos+3] == 'L')
					{
						if (m_nCharsUsed - m_nCurPos < 9)
						{
							goto ReadData;
						}
						if (m_pChars[m_nCurPos+4] != 'E' || m_pChars[m_nCurPos+5] != 'M' ||
							m_pChars[m_nCurPos+6] != 'E' || m_pChars[m_nCurPos+7] != 'N' || 
							m_pChars[m_nCurPos+8] != 'T')
						{
							SetLastError(XML_ExpectDtdMarkup);
							return TokenError;
						}
						m_nCurPos += 9;
						m_scanningFunction = ScanningFunctionQName;
						m_nextScaningFunction = ScanningFunctionElement1;
						return TokenElementDecl;
					}
					// <!ENTITY
					else if (m_pChars[m_nCurPos+3] == 'N')
					{
						if (m_nCharsUsed - m_nCurPos < 8)
						{
							goto ReadData;
						}
						if (m_pChars[m_nCurPos+4] != 'T' || m_pChars[m_nCurPos+5] != 'I' || 
							m_pChars[m_nCurPos+6] != 'T' || m_pChars[m_nCurPos+7] != 'Y')
						{
							SetLastError(XML_ExpectDtdMarkup);
							return TokenError;
						}
						m_nCurPos += 8;
						m_scanningFunction = ScanningFunctionEntity1;
						return TokenEntityDecl;
					}
					else
					{
						if (m_nCharsUsed - m_nCurPos < 4)
						{
							goto ReadData;
						}
						SetLastError(XML_ExpectDtdMarkup);
						return TokenError;
					}

				case 'A':
					// <!ATTLIST
					if (m_nCharsUsed - m_nCurPos < 9)
					{
						goto ReadData;
					}
					if (m_pChars[m_nCurPos+3] != 'T' || m_pChars[m_nCurPos+4] != 'T' || 
						m_pChars[m_nCurPos+5] != 'L' || m_pChars[m_nCurPos+6] != 'I' || 
						m_pChars[m_nCurPos+7] != 'S' || m_pChars[m_nCurPos+8] != 'T')
					{
						SetLastError(XML_ExpectDtdMarkup);
						return TokenError;
					}
					m_nCurPos += 9;
					m_scanningFunction = ScanningFunctionQName;
					m_nextScaningFunction = ScanningFunctionAttlist1;
					return TokenAttlistDecl;

				case 'N':
					// <!NOTATION
					if (m_nCharsUsed - m_nCurPos < 10)
					{
						goto ReadData;
					}
					if (m_pChars[m_nCurPos+3] != 'O' || m_pChars[m_nCurPos+4] != 'T' || 
						m_pChars[m_nCurPos+5] != 'A' || m_pChars[m_nCurPos+6] != 'T' || 
						m_pChars[m_nCurPos+7] != 'I' || m_pChars[m_nCurPos+8] != 'O' ||
						m_pChars[m_nCurPos+9] != 'N' )
					{
						SetLastError(XML_ExpectDtdMarkup);
						return TokenError;
					}
					m_nCurPos += 10;
					m_scanningFunction = ScanningFunctionName;
					m_nextScaningFunction = ScanningFunctionNotation1;
					return TokenNotationDecl;

				case '[':
					m_nCurPos += 3;
					m_scanningFunction = ScanningFunctionCondSection1;
					return TokenCondSectionStart;
				case '-':
					if (m_pChars[m_nCurPos+3] == '-')
					{
						m_nCurPos += 4;
						return TokenComment;
					}
					else
					{
						if (m_nCharsUsed - m_nCurPos < 4)
						{
							goto ReadData;
						}
						SetLastError(XML_ExpectDtdMarkup);
						return TokenError;
					}
				default:
					if (m_nCharsUsed - m_nCurPos < 3)
					{
						goto ReadData;
					}
					SetLastError(XML_ExpectDtdMarkup, m_nCurPos + 2);
					return TokenError;
				}
				break;
			case '?':
				m_nCurPos += 2;
				return TokenPI;
			default:
				if (m_nCharsUsed - m_nCurPos < 2)
				{
					goto ReadData;
				}
				SetLastError(XML_ExpectDtdMarkup);
				return TokenError;
			}
			break;
		case ']':
			if (m_nCharsUsed - m_nCurPos < 2 && !m_pReaderAdapter->IsEof())
			{
				goto ReadData;
			}
			if (m_pChars[m_nCurPos+1] != ']')
			{
				m_nCurPos++;  
				m_scanningFunction = ScanningFunctionClosingTag;
				return TokenRightBracket;
			}
			if (m_nCharsUsed - m_nCurPos < 3 && !m_pReaderAdapter->IsEof())
			{
				goto ReadData;
			}
			if (m_pChars[m_nCurPos+1] == ']' && m_pChars[m_nCurPos+2] == '>')
			{
				m_nCurPos += 3;
				return TokenCondSectionEnd;
			}
			/* no break */
		default:
			if (m_nCharsUsed - m_nCurPos == 0)
			{
				goto ReadData;
			}
			SetLastError(XML_ExpectDtdMarkup);
			return TokenError;
		}
ReadData:
		if (ReadData() == 0)
		{
			SetLastError(XML_IncompleteDtdContent, m_nCharsUsed);
			return TokenError;
		}
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return TokenError;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanElement1()
{
	for (;;)
	{
		switch (m_pChars[m_nCurPos])
		{
		case '(':
			m_scanningFunction = ScanningFunctionElement2;
			m_nCurPos++;
			return TokenLeftParen;
		case 'E':
			// <!EMPTY
			if (m_nCharsUsed - m_nCurPos < 5)
			{
				goto ReadData;
			}
			if (m_pChars[m_nCurPos+1] == 'M' && m_pChars[m_nCurPos+2] == 'P' &&
				m_pChars[m_nCurPos+3] == 'T' && m_pChars[m_nCurPos+4] == 'Y')
			{
				m_nCurPos += 5;
				m_scanningFunction = ScanningFunctionClosingTag;
				return TokenEMPTY;
			}
			goto Default;
		case 'A':
			// <!ANY
			if (m_nCharsUsed - m_nCurPos < 3)
			{
				goto ReadData;
			}
			if (m_pChars[m_nCurPos+1] == 'N' && m_pChars[m_nCurPos+2] == 'Y')
			{
				m_nCurPos += 3;
				m_scanningFunction = ScanningFunctionClosingTag;
				return TokenANY;
			}
			goto Default;
		default:
Default:
			SetLastError(XML_InvalidContentModel);
			break;
		}
ReadData:
		if (ReadData() == 0)
		{
			SetLastError(XML_IncompleteDtdContent);
			return TokenError;
		}
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return TokenError;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanElement2()
{
	if (m_pChars[m_nCurPos] == '#')
	{
		while (m_nCharsUsed - m_nCurPos < 7)
		{
			if (ReadData() == 0)
			{
				SetLastError(XML_IncompleteDtdContent);
				return TokenError;
			}
		}
		// #PCDATA
		if (m_pChars[m_nCurPos+1] == 'P' && m_pChars[m_nCurPos+2] == 'C' &&
			m_pChars[m_nCurPos+3] == 'D' && m_pChars[m_nCurPos+4] == 'A' &&
			m_pChars[m_nCurPos+5] == 'T' && m_pChars[m_nCurPos+6] == 'A')
		{
			m_nCurPos += 7;
			m_scanningFunction = ScanningFunctionElement6;
			return TokenPCDATA;
		}
		else
		{
			SetLastError(XML_ExpectPcData, m_nCurPos + 1);
			return TokenError;
		}
	}

	m_scanningFunction = ScanningFunctionElement3;
	return TokenNone;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanElement3()
{
	switch (m_pChars[m_nCurPos])
	{
	case '(':
		m_nCurPos++;
		return TokenLeftParen;
	case '>':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionSubsetContent;
		return TokenGreaterThan;
	default:
		ScanQName();
		m_scanningFunction = ScanningFunctionElement4;
		return TokenQName;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanElement4()
{
	m_scanningFunction = ScanningFunctionElement5;

	Token t;
	switch (m_pChars[m_nCurPos])
	{
	case '*':
		t = TokenStar;
		break;
	case '?':
		t = TokenQMark;
		break;
	case '+':
		t = TokenPlus;
		break;
	default:
		return TokenNone;
	}
	if (m_bWhitespaceBeforeToken)
	{
		SetLastError(XML_ExpectNoWhitespace);
		return TokenError;
	}
	m_nCurPos++;
	return t;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanElement5()
{
	switch (m_pChars[m_nCurPos] )
	{
	case ',':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionElement3;
		return TokenComma;
	case '|':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionElement3;
		return TokenOr;
	case ')':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionElement4;
		return TokenRightParen;
	case '>':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionSubsetContent;
		return TokenGreaterThan;
	default:
		SetLastError(XML_ExpectOp);
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanElement6()
{
	switch (m_pChars[m_nCurPos])
	{
	case ')':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionElement7;
		return TokenRightParen;
	case '|':
		m_nCurPos++;
		m_nextScaningFunction = ScanningFunctionElement6;
		m_scanningFunction = ScanningFunctionQName;
		return TokenOr;
	default:
		SetLastError(XML_UnexpectedToken, "),|");
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanElement7()
{
	m_scanningFunction = ScanningFunctionClosingTag;
	if (m_pChars[m_nCurPos] == '*' && !m_bWhitespaceBeforeToken)
	{
		m_nCurPos++;
		return TokenStar;
	}
	return TokenNone;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanAttlist1()
{
	switch (m_pChars[m_nCurPos])
	{
	case '>':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionSubsetContent;
		return TokenGreaterThan;
	default:
		if (!m_bWhitespaceBeforeToken)
		{
			SetLastError(XML_ExpectingWhiteSpace, CSTR(ParseUnexpectedToken(m_nCurPos)));
			return TokenError;
		}
		ScanQName();
		m_scanningFunction = ScanningFunctionAttlist2;
		return TokenQName;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanAttlist2()
{
	for (;;)
	{
		switch (m_pChars[m_nCurPos])
		{
		case '(':
			m_nCurPos++;
			m_scanningFunction = ScanningFunctionNmtoken;
			m_nextScaningFunction = ScanningFunctionAttlist5;
			return TokenLeftParen;
		case 'C':
			// CDATA
			if (m_nCharsUsed - m_nCurPos < 5)
				goto ReadData;
			if (m_pChars[m_nCurPos+1] != 'D' || m_pChars[m_nCurPos+2] != 'A' ||
				m_pChars[m_nCurPos+3] != 'T' || m_pChars[m_nCurPos+4] != 'A')
			{
				SetLastError(XML_InvalidAttributeType);
				return TokenError;
			}
			m_nCurPos += 5;
			m_scanningFunction = ScanningFunctionAttlist6;
			return TokenCDATA;
		case 'E':
			// ENTITY | EENTITIES
			if (m_nCharsUsed - m_nCurPos < 9)
				goto ReadData;
			m_scanningFunction = ScanningFunctionAttlist6;
			if (m_pChars[m_nCurPos+1] != 'N' || m_pChars[m_nCurPos+2] != 'T' ||
				m_pChars[m_nCurPos+3] != 'I' || m_pChars[m_nCurPos+4] != 'T')
			{
				SetLastError(XML_InvalidAttributeType);
				return TokenError;
			}
			switch (m_pChars[m_nCurPos+5])
			{
			case 'I':
				if (m_pChars[m_nCurPos+6] != 'E' || m_pChars[m_nCurPos+7] != 'S')
				{
					SetLastError(XML_InvalidAttributeType);
					return TokenError;
				}
				m_nCurPos += 8;
				return TokenENTITIES;
			case 'Y':
				m_nCurPos += 6;
				return TokenENTITY;
			default:
				SetLastError(XML_InvalidAttributeType);
				return TokenError;
			}
			break;
		case 'I':
			// ID | IDEF | IDEFS
			if (m_nCharsUsed - m_nCurPos < 6)
				goto ReadData;
			m_scanningFunction = ScanningFunctionAttlist6;
			if (m_pChars[m_nCurPos+1] != 'D')
			{
				SetLastError(XML_InvalidAttributeType);
				return TokenError;
			}
			if (m_pChars[m_nCurPos+2] != 'R')
			{
				m_nCurPos += 2;
				return TokenID;
			}
			if (m_pChars[m_nCurPos+3] != 'E' || m_pChars[m_nCurPos+4] != 'F')
			{
				SetLastError(XML_InvalidAttributeType);
				return TokenError;
			}
			if (m_pChars[m_nCurPos+5] != 'S')
			{
				m_nCurPos += 5;
				return TokenIDREF;
			}
			else
			{
				m_nCurPos += 6;
				return TokenIDREFS;
			}
		case 'N':
			// NOTATION | NMTOKEN | NMTOKENS
			if (m_nCharsUsed - m_nCurPos < 8 && !m_pReaderAdapter->IsEof())
			{
				goto ReadData;
			}
			switch (m_pChars[m_nCurPos+1])
			{
			case 'O':
				if (m_pChars[m_nCurPos+2] != 'T' || m_pChars[m_nCurPos+3] != 'A' || 
					m_pChars[m_nCurPos+4] != 'T' || m_pChars[m_nCurPos+5] != 'I' || 
					m_pChars[m_nCurPos+6] != 'O' || m_pChars[m_nCurPos+7] != 'N')
				{
					SetLastError(XML_InvalidAttributeType);
					return TokenError;
				}
				m_nCurPos += 8;
				m_scanningFunction = ScanningFunctionAttlist3;
				return TokenNOTATION;
			case 'M':
				if (m_pChars[m_nCurPos+2] != 'T' || m_pChars[m_nCurPos+3] != 'O' || 
					m_pChars[m_nCurPos+4] != 'K' || m_pChars[m_nCurPos+5] != 'E' || 
					m_pChars[m_nCurPos+6] != 'N')
				{
					SetLastError(XML_InvalidAttributeType);
					return TokenError;
				}
				m_scanningFunction = ScanningFunctionAttlist6;

				if (m_pChars[m_nCurPos+7] == 'S')
				{
					m_nCurPos += 8;
					return TokenNMTOKENS;
				}
				else
				{
					m_nCurPos += 7;
					return TokenNMTOKEN;
				}
			default:
				SetLastError(XML_InvalidAttributeType);
				return TokenError;
			}
			break;
		default:
			SetLastError(XML_InvalidAttributeType);
			return TokenError;
		}

ReadData:
		if (ReadData() == 0)
		{
			SetLastError(XML_IncompleteDtdContent);
			return TokenError;
		}
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return TokenError;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanAttlist3()
{
	if (m_pChars[m_nCurPos] == '(')
	{
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionName;
		m_nextScaningFunction = ScanningFunctionAttlist4;
		return TokenLeftParen;
	}
	else
	{
		SetLastError(XML_UnexpectedToken, '(');
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanAttlist4()
{
	switch (m_pChars[m_nCurPos])
	{
	case ')':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionAttlist6;
		return TokenRightParen;
	case '|':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionName;
		m_nextScaningFunction = ScanningFunctionAttlist4;
		return TokenOr;
	default:
		SetLastError(XML_UnexpectedToken, "(, |");
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanAttlist5()
{
	switch (m_pChars[m_nCurPos])
	{
	case ')':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionAttlist6;
		return TokenRightParen;
	case '|':
		m_nCurPos++;
		m_scanningFunction = ScanningFunctionNmtoken;
		m_nextScaningFunction = ScanningFunctionAttlist5;
		return TokenOr;
	default:
		SetLastError(XML_UnexpectedToken, "), |");
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanAttlist6()
{
	for (;;)
	{
		switch (m_pChars[m_nCurPos])
		{
		case '"':
		case '\'':
			if (ScanLiteral(LiteralTypeAttributeValue) == TokenError)
				return TokenError;
			m_scanningFunction = ScanningFunctionAttlist1;
			return TokenLiteral;
		case '#':
			// #REQUIRED | #IMPLIED | #FIXED
			if (m_nCharsUsed - m_nCurPos < 6)
				goto ReadData;
			switch (m_pChars[m_nCurPos+1])
			{
			case 'R':
				if (m_nCharsUsed - m_nCurPos < 9)
					goto ReadData;
				if (m_pChars[m_nCurPos+2] != 'E' || m_pChars[m_nCurPos+3] != 'Q' ||
					m_pChars[m_nCurPos+4] != 'U' || m_pChars[m_nCurPos+5] != 'I' ||
					m_pChars[m_nCurPos+6] != 'R' || m_pChars[m_nCurPos+7] != 'E' ||
					m_pChars[m_nCurPos+8] != 'D')
				{
					SetLastError(XML_ExpectAttType);
					return TokenError;
				}
				m_nCurPos += 9;
				m_scanningFunction = ScanningFunctionAttlist1;
				return TokenREQUIRED;
			case 'I':
				if (m_nCharsUsed - m_nCurPos < 8)
					goto ReadData;
				if (m_pChars[m_nCurPos+2] != 'M' || m_pChars[m_nCurPos+3] != 'P' ||
					m_pChars[m_nCurPos+4] != 'L' || m_pChars[m_nCurPos+5] != 'I' ||
					m_pChars[m_nCurPos+6] != 'E' || m_pChars[m_nCurPos+7] != 'D')
				{
					SetLastError(XML_ExpectAttType);
					return TokenError;
				}
				m_nCurPos += 8;
				m_scanningFunction = ScanningFunctionAttlist1;
				return TokenIMPLIED;
			case 'F':
				if (m_pChars[m_nCurPos+2] != 'I' || m_pChars[m_nCurPos+3] != 'X' ||
					m_pChars[m_nCurPos+4] != 'E' || m_pChars[m_nCurPos+5] != 'D')
				{
					SetLastError(XML_ExpectAttType);
					return TokenError;
				}
				m_nCurPos += 6;
				m_scanningFunction = ScanningFunctionAttlist7;
				return TokenFIXED;
			default:
				SetLastError(XML_ExpectAttType);
				return TokenError;
			}
			break;
		default:
			SetLastError(XML_ExpectAttType);
			return TokenError;
		}
ReadData:
		if (ReadData() == 0)
		{
			SetLastError(XML_IncompleteDtdContent);
			return TokenError;
		}
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return TokenError;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanAttlist7()
{
	switch (m_pChars[m_nCurPos])
	{
	case '"':
	case '\'':
		if (ScanLiteral(LiteralTypeAttributeValue) == TokenError)
			return TokenError;
		m_scanningFunction = ScanningFunctionAttlist1;
		return TokenLiteral;
	default:
		BFX_ASSERT1(false, "We should never get to this point.");
		SetLastError(XML_UnexpectedToken, "\",'");
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanNotation1()
{
	switch (m_pChars[m_nCurPos])
	{
	case 'P':
		if (!EatPublicKeyword())
		{
			SetLastError(XML_ExpectExternalOrClose);
			return TokenError;
		}
		m_nextScaningFunction = ScanningFunctionClosingTag;
		m_scanningFunction = ScanningFunctionPublicId1;
		return TokenPUBLIC;
	case 'S':
		if (!EatSystemKeyword())
		{
			SetLastError(XML_ExpectExternalOrClose);
			return TokenError;
		}
		m_nextScaningFunction = ScanningFunctionClosingTag;
		m_scanningFunction = ScanningFunctionSystemId;
		return TokenSYSTEM;
	default:
		SetLastError(XML_ExpectExternalOrPublicId);
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanSystemId()
{
	if (m_pChars[m_nCurPos] != '"' && m_pChars[m_nCurPos] != '\'')
	{
		SetLastError(XML_UnexpectedToken, "\",'");
		return TokenError;
	}

	ScanLiteral(LiteralTypeSystemOrPublicID);

	m_scanningFunction = m_nextScaningFunction;
	return TokenLiteral;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanPublicId1()
{
	if (m_pChars[m_nCurPos] != '"' && m_pChars[m_nCurPos] != '\'')
	{
		SetLastError(XML_UnexpectedToken, "\",'");
		return TokenError;
	}

	ScanLiteral(LiteralTypeSystemOrPublicID);

	m_scanningFunction = ScanningFunctionPublicId2;
	return TokenLiteral;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanPublicId2()
{
	if (m_pChars[m_nCurPos] != '"' && m_pChars[m_nCurPos] != '\'')
	{
		m_scanningFunction = m_nextScaningFunction;
		return TokenNone;
	}

	ScanLiteral(LiteralTypeSystemOrPublicID);
	m_scanningFunction = m_nextScaningFunction;

	return TokenLiteral;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanEntity1()
{
	if (m_pChars[m_nCurPos] == '%')
	{
		m_nCurPos++;
		m_nextScaningFunction = ScanningFunctionEntity2;
		m_scanningFunction = ScanningFunctionName;
		return TokenPercent;
	}
	else
	{
		ScanName();
		m_scanningFunction = ScanningFunctionEntity2;
		return TokenName;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanEntity2()
{
	switch (m_pChars[m_nCurPos])
	{
	case 'P':
		if (!EatPublicKeyword())
		{
			SetLastError(XML_ExpectExternalOrClose);
			return TokenError;
		}
		m_nextScaningFunction = ScanningFunctionEntity3;
		m_scanningFunction = ScanningFunctionPublicId1;
		return TokenPUBLIC;
	case 'S':
		if (!EatSystemKeyword())
		{
			SetLastError(XML_ExpectExternalOrClose);
			return TokenError;
		}
		m_nextScaningFunction = ScanningFunctionEntity3;
		m_scanningFunction = ScanningFunctionSystemId;
		return TokenSYSTEM;

	case '"':
	case '\'':
		ScanLiteral(LiteralTypeEntityReplText);
		m_scanningFunction = ScanningFunctionClosingTag;
		return TokenLiteral;
	default:
		SetLastError(XML_ExpectExternalIdOrEntityValue);
		return TokenError;
	}
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanEntity3()
{
	if (m_pChars[m_nCurPos] == 'N')
	{
		while (m_nCharsUsed - m_nCurPos < 5)
		{
			if (ReadData() == 0)
			{
				goto End;
			}
		}
		if (m_pChars[m_nCurPos+1] == 'D' && m_pChars[m_nCurPos+2] == 'A' && 
			m_pChars[m_nCurPos+3] == 'T' && m_pChars[m_nCurPos+4] == 'A')
		{
			m_nCurPos += 5;
			m_scanningFunction = ScanningFunctionName;
			m_nextScaningFunction = ScanningFunctionClosingTag;
			return TokenNData;
		}
	}
End:
	m_scanningFunction = ScanningFunctionClosingTag;
	return TokenNone;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanCondSection1()
{
	if (m_pChars[m_nCurPos] != 'I')
	{
		SetLastError(XML_ExpectIgnoreOrInclude);
		return TokenError;
	}
	m_nCurPos++;

	for (;;)
	{
		if (m_nCharsUsed - m_nCurPos < 5)
		{ 
			goto ReadData;
		}
		switch (m_pChars[m_nCurPos])
		{
		case 'N':
			// INCLUDE
			if (m_nCharsUsed - m_nCurPos < 6)
			{ 
				goto ReadData;
			}
			if (m_pChars[m_nCurPos+1] != 'C' || m_pChars[m_nCurPos+2] != 'L' ||
				m_pChars[m_nCurPos+3] != 'U' || m_pChars[m_nCurPos+4] != 'D' || 
				m_pChars[m_nCurPos+5] != 'E' || XMLCharType::IsNameChar(m_pChars[m_nCurPos+6]))
			{
				goto Default;
			}
			m_nextScaningFunction = ScanningFunctionSubsetContent;
			m_scanningFunction = ScanningFunctionCondSection2;
			m_nCurPos += 6;
			return TokenINCLUDE;
		case 'G':
			// IGNORE
			if (m_pChars[m_nCurPos+1] != 'N' || m_pChars[m_nCurPos+2] != 'O' ||
				m_pChars[m_nCurPos+3] != 'R' || m_pChars[m_nCurPos+4] != 'E' ||
				XMLCharType::IsNameChar(m_pChars[m_nCurPos+5]))
			{
				goto Default;
			}
			m_nextScaningFunction = ScanningFunctionCondSection3;
			m_scanningFunction = ScanningFunctionCondSection2;
			m_nCurPos += 5;
			return TokenIGNORE;
		default:
Default:
			SetLastError(XML_ExpectIgnoreOrInclude, m_nCurPos - 1);
			return TokenNone;
		}
ReadData:
		if (ReadData() == 0)
		{
			SetLastError(XML_IncompleteDtdContent);
			return TokenError;
		}
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return TokenError;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanCondSection2()
{
	if (m_pChars[m_nCurPos] != '[')
	{
		SetLastError(XML_UnexpectedToken, "[");
		return TokenError;
	}
	m_nCurPos++;
	m_scanningFunction = m_nextScaningFunction;
	return TokenRightBracket;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanCondSection3()
{
	int nIgnoreSectionDepth = 0;

	// skip ignored part
	for (;;)
	{
		while (XMLCharType::IsTextChar(m_pChars[m_nCurPos]) && m_pChars[m_nCurPos] != ']')
		{
			m_nCurPos++;
		}

		switch (m_pChars[m_nCurPos])
		{
		case '"':
		case '\'':
		case 0x9:
		case '&':
			m_nCurPos++;
			continue;
			// eol
		case 0xA:
			m_nCurPos++;
			m_pReaderAdapter->OnNewLine(m_nCurPos);
			continue;
		case 0xD:
			BFX_ASSERT(!IsParsingInternalSubset());	// no need to normalize
			if (m_pChars[m_nCurPos+1] == 0xA)
			{
				m_nCurPos += 2;
			}
			else if (m_nCurPos+1 < m_nCharsUsed || m_pReaderAdapter->IsEof())
			{
				m_nCurPos++;
			}
			else {
				goto ReadData;
			}
			m_pReaderAdapter->OnNewLine(m_nCurPos);
			continue;
		case '<':
			if (m_nCharsUsed - m_nCurPos < 3)
			{
				goto ReadData;
			}
			if (m_pChars[m_nCurPos+1] != '!' || m_pChars[m_nCurPos+2] != '[')
			{
				m_nCurPos++;
				continue;
			}
			nIgnoreSectionDepth++;
			m_nCurPos += 3;
			continue;
		case ']':
			if (m_nCharsUsed - m_nCurPos < 3)
			{
				goto ReadData;
			}
			if (m_pChars[m_nCurPos+1] != ']' || m_pChars[m_nCurPos+2] != '>')
			{
				m_nCurPos++;
				continue;
			}
			if (nIgnoreSectionDepth > 0)
			{
				nIgnoreSectionDepth--;
				m_nCurPos += 3;
				continue;
			}
			else
			{
				m_nCurPos += 3;
				m_scanningFunction = ScanningFunctionSubsetContent;
				return TokenCondSectionEnd;
			}
		default:
			// end of buffer
			if (m_nCurPos == m_nCharsUsed)
			{
				goto ReadData;
			}
			// surrogate chars
			else { 
				UCHAR ch = m_pChars[m_nCurPos];
				if (CharUnicodeInfo::IsHighSurrogate(ch))
				{
					if (m_nCurPos + 1 == m_nCharsUsed)
					{
						goto ReadData;
					}
					m_nCurPos++;
					if (CharUnicodeInfo::IsLowSurrogate(m_pChars[m_nCurPos]))
					{
						m_nCurPos++;
						continue;
					}
				}
				SetLastError(XML_InvalidCharacter, String((char)ch));
				return TokenNone;
			}
		}

ReadData:
		// read new characters into the buffer
		if (m_pReaderAdapter->IsEof() || ReadData() == 0)
		{
			if (HandleEntityEnd( false ))
			{
				continue;
			}
			SetLastError(XML_UnclosedConditionalSection);
			return TokenError;
		}
		m_nTokenStartPos = m_nCurPos;
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return TokenError;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanLiteral(LiteralType literalType)
{
	BFX_ASSERT(m_pChars[m_nCurPos] == '"' || m_pChars[m_nCurPos] == '\'');

	UCHAR quoteChar = m_pChars[m_nCurPos];
	UCHAR replaceChar = (literalType == LiteralTypeAttributeValue) ? 0x20 : 0xA;
	int nStartQuoteEntityId = m_nCurEntityId;

	// TODO: set literal line info
	m_nCurPos++;
	m_nTokenStartPos = m_nCurPos;

	m_pStringBuilder->SetLength(0);

	for (;;)
	{
		while (XMLCharType::IsAttributeValueChar(m_pChars[m_nCurPos]) && m_pChars[m_nCurPos] != '%')
		{
			m_nCurPos++;
		}

		if (m_pChars[m_nCurPos] == quoteChar && m_nCurEntityId == nStartQuoteEntityId)
		{
			if (m_pStringBuilder->GetLength() > 0)
			{
				m_pStringBuilder->Append(m_pChars, m_nTokenStartPos, m_nCurPos - m_nTokenStartPos);
			}
			m_nCurPos++;
			m_literalQuoteChar = quoteChar;
			return TokenLiteral;
		}

		int tmp1 = m_nCurPos - m_nTokenStartPos;
		if (tmp1 > 0)
		{
			m_pStringBuilder->Append(m_pChars, m_nTokenStartPos, tmp1);
			m_nTokenStartPos = m_nCurPos;
		}

		switch (m_pChars[m_nCurPos])
		{
		case '"':
		case '\'':
		case '>':
			m_nCurPos++;
			continue;
			// eol
		case 0xA:
			m_nCurPos++;
			if (m_bNormalize)
			{
				m_pStringBuilder->Append(replaceChar);	// For attributes: CDATA normalization of 0xA
				m_nTokenStartPos = m_nCurPos;
			}
			m_pReaderAdapter->OnNewLine(m_nCurPos);
			continue;
		case 0xD:
			if (m_pChars[m_nCurPos+1] == 0xA)
			{
				if (m_bNormalize)
				{
					if (literalType == LiteralTypeAttributeValue)
					{
						m_pStringBuilder->Append(0x20);	// CDATA normalization of 0xD 0xA
					}
					else
					{
						m_pStringBuilder->Append(0x0A);	// EOL normalization of 0xD 0xA									
					}
				}
				m_nTokenStartPos = m_nCurPos + 2;

				SaveParsingBuffer();	// EOL normalization of 0xD 0xA in internal subset value
				m_pReaderAdapter->SetCurrentPosition(m_pReaderAdapter->GetCurrentPosition() + 1);

				m_nCurPos += 2;
			}
			else if (m_nCurPos+1 == m_nCharsUsed)
			{
				goto ReadData;
			}
			else
			{
				m_nCurPos++;
				if (m_bNormalize)
				{
					m_pStringBuilder->Append(replaceChar); // For attributes: CDATA normalization of 0xD and 0xD 0xA
					m_nTokenStartPos = m_nCurPos;              // For entities:   EOL normalization of 0xD and 0xD 0xA
				}
			}
			m_pReaderAdapter->OnNewLine(m_nCurPos);
			continue;
			// tab
		case 0x9:
			if (literalType == LiteralTypeAttributeValue && m_bNormalize)
			{
				m_pStringBuilder->Append(0x20);      // For attributes: CDATA normalization of 0x9
				m_nTokenStartPos++;
			}
			m_nCurPos++;
			continue;
			// attribute values cannot contain '<'
		case '<':
			if (literalType == LiteralTypeAttributeValue)
			{
				SetLastError(XML_BadAttributeChar, '<');
				return TokenError;
			}
			m_nCurPos++;
			continue;
			// parameter entity reference
		case '%':
			if (literalType != LiteralTypeEntityReplText)
			{
				m_nCurPos++;
				continue;
			}
			HandleEntityReference(true, true, literalType == LiteralTypeAttributeValue);
			m_nTokenStartPos = m_nCurPos;
			continue;
			// general entity reference
		case '&':
			if (literalType == LiteralTypeSystemOrPublicID)
			{
				m_nCurPos++;
				continue;
			}
			if (m_nCurPos + 1 == m_nCharsUsed)
			{
				goto ReadData;
			}
			// numeric characters reference
			if (m_pChars[m_nCurPos + 1] == '#')
			{
				SaveParsingBuffer();
				int nEndPos = m_pReaderAdapter->ParseNumericCharRef(IsSaveInternalSubsetValue() ? m_pInternalSubsetValueSb : NULL);
				LoadParsingBuffer();
				m_pStringBuilder->Append(m_pChars, m_nCurPos, nEndPos - m_nCurPos);
				m_pReaderAdapter->SetCurrentPosition(nEndPos);
				m_nTokenStartPos = nEndPos;
				m_nCurPos = nEndPos;
				continue;
			}
			else {
				// general entity reference
				SaveParsingBuffer();
				if ( literalType == LiteralTypeAttributeValue )
				{
					int nEndPos = m_pReaderAdapter->ParseNamedCharRef(true, IsSaveInternalSubsetValue() ? m_pInternalSubsetValueSb : NULL);
					LoadParsingBuffer();

					if (nEndPos >= 0)
					{
						m_pStringBuilder->Append(m_pChars, m_nCurPos, nEndPos - m_nCurPos);
						m_pReaderAdapter->SetCurrentPosition(nEndPos);
						m_nTokenStartPos = nEndPos;
						m_nCurPos = nEndPos;
						continue;
					}
					else
					{
						HandleEntityReference(false, true, true);
						m_nTokenStartPos = m_nCurPos;
					}
					continue;
				}
				else
				{
					int nEndPos = m_pReaderAdapter->ParseNamedCharRef(false, NULL);
					LoadParsingBuffer();

					if (nEndPos >= 0)
					{
						m_nTokenStartPos = m_nCurPos;
						m_nCurPos = nEndPos;
						continue;
					}
					else
					{
						m_pStringBuilder->Append('&');
						m_nCurPos++;
						m_nTokenStartPos = m_nCurPos;
						// Bypass general entities in entity values
						REFXMLQualifiedName* pEntityName = ScanEntityName();
						VerifyEntityReference(pEntityName, false, false, false);
					}
					continue;
				}
			}
		default:
			// end of buffer
			if (m_nCurPos == m_nCharsUsed)
			{
				goto ReadData;
			}
			// surrogate chars
			else { 
				UCHAR ch = m_pChars[m_nCurPos];
				if (CharUnicodeInfo::IsHighSurrogate(ch))
				{
					if (m_nCurPos + 1 == m_nCharsUsed)
					{
						goto ReadData;
					}
					m_nCurPos++;
					if (CharUnicodeInfo::IsLowSurrogate(m_pChars[m_nCurPos]))
					{
						m_nCurPos++;
						continue;
					}
				}
				SetLastError(XML_InvalidCharacter, ch);
				return TokenError;
			}
		}

ReadData:
		BFX_ASSERT(m_nCurPos - m_nTokenStartPos == 0);

		// read new characters into the buffer
		if (m_pReaderAdapter->IsEof() || ReadData() == 0)
		{
			if (literalType == LiteralTypeSystemOrPublicID || !HandleEntityEnd(true))
			{
				SetLastError(XML_UnclosedQuote);
				return TokenError;
			}
		}
		m_nTokenStartPos = m_nCurPos;
	}
	BFX_ASSERT1(false, "Should never touches this point.");
	return TokenError;
}

REFXMLDtdParser::Token REFXMLDtdParser::ScanClosingTag()
{
	if (m_pChars[m_nCurPos] != '>')
	{
		SetLastError(XML_UnexpectedToken, ">");
		return TokenError;
	}
	m_nCurPos++;
	m_scanningFunction = ScanningFunctionSubsetContent;
	return TokenGreaterThan;
}

// Scans the entity name from current read position.
// Returns the qualified name if success. Returns NULL if error occurs, the errorcode been set.
REFXMLQualifiedName* REFXMLDtdParser::ScanEntityName()
{
	if (!ScanName())
	{
		// NOTE: errorcode has been replaced.
		SetLastError(XML_ErrorParsingEntityName);
		return NULL;
	}

	if (m_pChars[m_nCurPos] != ';')
	{
		SetLastError(XML_UnexpectedToken, ";");
		return NULL;
	}

	REFXMLQualifiedName* pEntityName = GetNameQualified(false);
	m_nCurPos ++;

	return pEntityName;
}

bool REFXMLDtdParser::EatPublicKeyword()
{
	BFX_ASSERT(m_pChars[m_nCurPos] == 'P');
	while (m_nCharsUsed - m_nCurPos < 6)
	{
		// end of buffer
		if (ReadData() == 0)
		{
			return false;
		}
	}
	// PUBLIC
	if (m_pChars[m_nCurPos+1] != 'U' || m_pChars[m_nCurPos+2] != 'B' ||
		m_pChars[m_nCurPos+3] != 'L' || m_pChars[m_nCurPos+4] != 'I' ||
		m_pChars[m_nCurPos+5] != 'C' )
	{
		return false;
	}
	m_nCurPos += 6;
	return true;
}

bool REFXMLDtdParser::EatSystemKeyword()
{ 
	BFX_ASSERT(m_pChars[m_nCurPos] == 'S');
	while (m_nCharsUsed - m_nCurPos < 6)
	{
		// end of buffer
		if (ReadData() == 0)
		{
			return false;
		}
	}
	// SYSTEM
	if (m_pChars[m_nCurPos+1] != 'Y' || m_pChars[m_nCurPos+2] != 'S' ||
		m_pChars[m_nCurPos+3] != 'T' || m_pChars[m_nCurPos+4] != 'E' ||
		m_pChars[m_nCurPos+5] != 'M' )
	{
		return false;
	}
	m_nCurPos += 6;
	return true;
}

UniString REFXMLDtdParser::GetValueWithStrippedSpaces()
{
    BFX_ASSERT(m_nCurPos == 0 || m_pChars[m_nCurPos-1] == '"' || m_pChars[m_nCurPos-1] == '\'');

	const UCHAR* pch;
	int iOffset, nLength;
    if (m_pStringBuilder->GetLength() == 0)
	{
		pch = m_pChars;
		iOffset = m_nTokenStartPos;
		nLength = m_nCurPos - m_nTokenStartPos - 1;
    }
    else
	{
		pch = m_pStringBuilder->GetData();
		iOffset = 0;
		nLength = m_pStringBuilder->GetLength();
    }

	// removes spaces at the beginning and at the end of the value and replaces sequences of spaces
	// with a single space
	if (nLength <= 0)
	{
		// the value is empty.
		return UniString();
	}

	int iStartPos = iOffset;
	int iEndPos = iOffset + nLength;

	// strips leading spaces.
	while (pch[iStartPos] == 0x20)
	{
		iStartPos++;
		if (iStartPos == iEndPos)
		{
			// the value are full space characters.
			return USTR(" ");
		}
	}

	UniString strResult;
	int i;
	for (i = iStartPos; i < iEndPos; i++)
	{
		UCHAR ch;
		if ((ch = pch[i] ) == 0x20)
		{
			int j = i + 1;
			// skiping space characters sequence.
			while (j < iEndPos && pch[j] == 0x20)
			{
				j++;
			}
			// at the end position.
			if (j == iEndPos)
			{
				strResult.Append(pch + iStartPos, i - iStartPos);
				break;
			}
			if (j > i+1)
			{
				strResult.Append(pch + iStartPos, i - iStartPos + 1);
				iStartPos = j;
				i = j - 1;
			}
		}
	}
	strResult.Append(pch + iStartPos, i - iStartPos);

	return strResult;
}

//
// Entity handling
//
bool REFXMLDtdParser::HandleEntityReference(REFXMLQualifiedName* pEntityName, bool bParamEntity, bool bInLiteral, bool bInAttribute)
{
	BFX_ASSERT(m_pChars[m_nCurPos - 1] == ';');

	SaveParsingBuffer();
	if (bParamEntity && IsParsingInternalSubset() && !IsParsingTopLevelMarkup())
	{
		SetLastError(XML_InvalidParEntityRef, m_nCurPos - pEntityName->GetName().GetLength() - 1);
		return false;
	}

	REFXMLSchemaEntity* pEntity = VerifyEntityReference(pEntityName, bParamEntity, true,
			bInAttribute);
	if (pEntity == NULL/* || IsErrorOccurred()*/)
	{
		return false;
	}
	if (pEntity->IsProcessed())
	{
		SetLastError(XML_RecursiveGenEntity, CSTR(pEntityName->GetName()), m_nCurPos - pEntityName->GetName().GetLength() - 1);
		return false;
	}

	int nNewEntityId = m_nNextEntityId++;

	if (pEntity->IsExternal())
	{
		if (!m_pReaderAdapter->PushEntity(pEntity, nNewEntityId))
		{
			return false;
		}
		m_nExternalEntitiesDepth++;
	}
	else
	{
		if (pEntity->GetText().GetLength() == 0)
		{
			return false;
		}
		if (!m_pReaderAdapter->PushEntity(pEntity, nNewEntityId))
		{
			return false;
		}
	}
	m_nCurEntityId = nNewEntityId;

	if (bParamEntity && !bInLiteral && m_scanningFunction != ScanningFunctionParamEntitySpace)
	{
		m_savedScanningFunction = m_scanningFunction;
		m_scanningFunction = ScanningFunctionParamEntitySpace;
	}

	LoadParsingBuffer();
	return true;
}

REFXMLSchemaEntity* REFXMLDtdParser::VerifyEntityReference(REFXMLQualifiedName* pEntityName, bool bParamEntity, bool bMustBeDeclared, bool bInAttribute)
{
	BFX_ASSERT(m_pChars[m_nCurPos - 1] == ';');

	REF < REFXMLSchemaEntity > pEntity;
	if (bParamEntity)
	{
		m_pSchemaInfo->GetParameterEntities()->Lookup(pEntityName, pEntity);
	}
	else
	{
		m_pSchemaInfo->GetGeneralEntities()->Lookup(pEntityName, pEntity);
	}

	if (pEntity == NULL)
	{
		if (bParamEntity)
		{
			if (m_bValidate)
			{
				// TODO: validate
				BFX_ASSERT(false);
			}
		}
		else if (bMustBeDeclared)
		{
			if (!IsParsingInternalSubset())
			{
				// TODO: validate
				BFX_ASSERT(false);
			}
			else
			{
				SetLastError(XML_UndeclaredEntity, CSTR(pEntityName->GetName()), m_nCurPos - pEntityName->GetName().GetLength() - 1);
			}
		}
		return NULL;
	}

	if (!pEntity->GetNData()->IsEmpty())
	{
		SetLastError(XML_UnparsedEntityRef, CSTR(pEntityName->GetName()), m_nCurPos - pEntityName->GetName().GetLength() - 1);
		return NULL;
	}

	if (bInAttribute && pEntity->IsExternal())
	{
		SetLastError(XML_ExternalEntityInAttValue, CSTR(pEntityName->GetName()), m_nCurPos - pEntityName->GetName().GetLength() - 1);
		return NULL;
	}

	return pEntity;
}

bool REFXMLDtdParser::HandleEntityEnd(bool bInLiteral)
{
	SaveParsingBuffer();

	REFXMLSchemaEntity* pOldEntity;
	if (!m_pReaderAdapter->PopEntity(OUT pOldEntity, OUT m_nCurEntityId))
	{
		return false;
	}
	LoadParsingBuffer();

	if (pOldEntity == NULL)
	{
		// external subset popped
		BFX_ASSERT(!IsParsingInternalSubset() || m_bFreeFloatingDtd);
		BFX_ASSERT(m_nCurEntityId == 0);
		if (m_scanningFunction == ScanningFunctionParamEntitySpace)
		{
			m_scanningFunction = m_savedScanningFunction;
		}
		return false;
	}

	if (pOldEntity->IsExternal())
	{
		m_nExternalEntitiesDepth--;
	}

	if (!bInLiteral && m_scanningFunction != ScanningFunctionParamEntitySpace)
	{
		m_savedScanningFunction = m_scanningFunction;
		m_scanningFunction = ScanningFunctionParamEntitySpace;
	}

	return true;
}

}	//	namespace BFX
